#!/bin/bash -l
#SBATCH --nodes=1
#SBATCH --time=2:00:00
#SBATCH -A e3sm
#SBATCH -p slurm
#SBATCH --qos regular
#SBATCH --job-name=mpi4py
#SBATCH --output=mpi4py.o%j
#SBATCH --error=mpi4py.e%j

set -e


source /compyfs/asay932/test_e3sm_unified/test_e3sm_unified_1.12.0rc3_compy.sh

python -c "from mpi4py import MPI"
python -c "from ILAMB.ModelResult import ModelResult"

--------------------------------------------------------------------------
By default, for Open MPI 4.0 and later, infiniband ports on a device
are not used by default.  The intent is to use UCX for these devices.
You can override this policy by setting the btl_openib_allow_ib MCA parameter
to true.

  Local host:              n0416
  Local adapter:           hfi1_0
  Local port:              1

--------------------------------------------------------------------------
--------------------------------------------------------------------------
WARNING: There was an error initializing an OpenFabrics device.

  Local host:   n0416
  Local device: hfi1_0
--------------------------------------------------------------------------

salloc --nodes 1 --qos interactive --time 00:15:00 --constraint cpu --account e3sm 

source /global/common/software/e3sm/anaconda_envs/test_e3sm_unified_1.12.0rc3_pm-cpu.sh

python -c "import mpi4py, xarray; print('mpi4py:', mpi4py.__version__)"
#mpi4py: 4.1.1 

srun -n 2 python -c "from mpi4py import MPI; print(MPI.COMM_WORLD.Get_size())"
# Traceback (most recent call last):                                                                                                                         
# Traceback (most recent call last):                                                                                                                         
#  File "<string>", line 1, in <module>                 
#     from mpi4py import MPI; print(MPI.COMM_WORLD.Get_size())                                                                                               
#     ^^^^^^^^^^^^^^^^^^^^^^                   
# ImportError: libmpi.so.12: cannot open shared object file: No such file or directory
#   File "<string>", line 1, in <module>
#     from mpi4py import MPI; print(MPI.COMM_WORLD.Get_size())                                                                                               
#     ^^^^^^^^^^^^^^^^^^^^^^

echo
echo ===== RUN ILAMB =====
echo

# Run diagnostics
# Not required TODO?
# TODO: find the mpi run format for different platforms

# include cfg file
cat > ilamb.cfg << EOF
{% include cfg %}
EOF

echo ${workdir}
echo {{ scriptDir }}

srun -N 1 ilamb-run --config ilamb.cfg --model_root $model_root  --regions global --model_year ${Y1} 2000